package exp.web.usatoday;

import java.util.ArrayList;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import tools.FileReaderEnhance;
import tools.twitter.TweetTidy;
import exp.URLContent;

/**
 * 提取USA Today的highlights和内容
 * 
 * @author Lanjun
 * 
 */
public class USATodayExtraction {
	// USA today highlights
	static final Pattern UTHL = Pattern
			.compile("<li class=\"hlt-item\">(.*?)</li>");
	// USA today paragraphs
	static final Pattern UTP = Pattern.compile("<p>(.*?)</p>");
	// USA today Title
	static final Pattern UT = Pattern.compile("<title>(.*?)</title>");
	
	//highlights tidy patterns
	static final Pattern [] hlPatterns = {
		Pattern.compile("</?[^>]+>"),
	};
	//content tidy patterns
	static final Pattern [] cPatterns = {
		Pattern.compile("</?[^>]+>"),
	};

	public static void main(String[] args) {
		URLContent hc1 = getUSAToday(
				FileReaderEnhance.readToString("data/test/usatoday1.htm"),
				"test1");
		URLContent hc2 = getUSAToday(
				FileReaderEnhance.readToString("data/test/usatoday2.htm"),
				"test2");
		URLContent hc3 = getUSAToday(
				FileReaderEnhance.readToString("data/test/usatoday3_nh.htm"),
				"test3");
		System.out.println(hc1.title);
		System.out.println(hc2.highlights);
		System.out.println(hc3.url);
	}

	public static URLContent getUSAToday(String htmlContent, String url) {
		URLContent hc = new URLContent();
		hc.url = url;
		if (htmlContent == null) {
			return hc;
		}

		// get title
		Matcher tm = UT.matcher(htmlContent);
		while (tm.find()) {
			String title = tm.group(1);
			hc.title = title;
		}

		hc.contents = new ArrayList<>();
		int hStart = htmlContent
				.indexOf("<h3 class=\"hlts\">Story Highlights</h3>");
		if (hStart == -1) {
			// no highlights
			hc.highlights = null;
		} else {
			hc.highlights = new ArrayList<>();
			int hEnd = htmlContent.indexOf("</ul></aside>", hStart + 38);
			String hl = htmlContent.substring(hStart + 38, hEnd);
			Matcher m = UTHL.matcher(hl);
			while (m.find()) {
				String oneHL = m.group(1);
				hc.highlights.add(oneHL);
			}
		}
		if (hc.highlights == null) {
			return hc;
		}
		// get the contents
		int cStart = htmlContent.indexOf("</aside></div></div><p>");
		int cEnd = htmlContent.indexOf("</div><aside ", cStart + 23);
		// to preserve <p>
		cStart = cStart + 20;
		String ch = htmlContent.substring(cStart, cEnd);
		Matcher m = UTP.matcher(ch);
		while (m.find()) {
			String oneP = m.group(1);
			hc.contents.add(oneP);
		}

		return hc;
	}

	public static String unify(String url) {
		int position = url.lastIndexOf('/');
		return url.substring(0, position + 1);
	}
	
	public static URLContent tidy(URLContent uc) {
		for (int i = 0; i < uc.highlights.size(); i++) {
			String hl = uc.highlights.get(i);
			hl = TweetTidy.doTidyHTML(hl);
			for (Pattern p : hlPatterns) {
				hl = p.matcher(hl).replaceAll("");
			}
			uc.highlights.set(i, hl);
		}
		for (int i = 0; i < uc.contents.size(); i++) {
			String c = uc.contents.get(i);
			c = TweetTidy.doTidyHTML(c);
			for (Pattern p : cPatterns) {
				c = p.matcher(c).replaceAll("");
			}
			uc.contents.set(i, c);
		}
		
		return uc;
	}
}
